#-*- codeing = utf-8 -*-
#@Time : 2020/10/21 15:57
#@Author : 阳某
#@File : Python自动翻译英语论文PDF.py
#@Software : PyCharm

'''
Python自动翻译英语论文PDF
涉及技术：

Python读取PDF文本
pandas的读取csv、多数据merge、输出Excel
Python正则表达式实现英文分词
'''
import pdfplumber
import pandas as pd

def read_pdf(pdf_fpath):
    pdf = pdfplumber.open(pdf_fpath)
    page_conts = []
    for page in pdf.pages:
        page_conts.append(page.extract_text())
    pdf.close()
    return " ".join(page_conts)
pdf_fpath = "D:/tmp/Wide & Deep Learning for Recommender Systems.pdf"
pdf_cont = read_pdf(pdf_fpath)


# 2. 读取英语-汉语翻译词典文件
# 词典文件来自：https://github.com/skywind3000/ECDICT 使用步骤：
#
# 下载代码打包：https://github.com/skywind3000/ECDICT/archive/master.zip
# 解压master.zip，然后解压其中的‪stardict.csv文件
# 注意：stardict.csv的地址需要替换成你自己的文件地址
df_dict = pd.read_csv("D:/tmp/ECDICT-master/stardict.csv")
print(df_dict.shape)
print(df_dict.sample(10).head())

# 把word、translation之外的列扔掉
df_dict = df_dict[["word", "translation"]]
print(df_dict.head())

# 3. 英文分词和数据清洗

# 分词
import re
word_list = re.split("""[ ,.\(\)/\n|\-:=\$\["']""", pdf_cont)
print(word_list[:10])

# 数据清洗
word_list_clean = []
for word in word_list:
    word = str(word).lower().strip()
    # 过滤掉空词、数字、单个字符的词、停用词
    if not word or word.isnumeric() or len(word)<=1:
        continue
    word_list_clean.append(word)
print(word_list_clean[:20])
# 4. 分词结果构造成一个DataFrame

df_words = pd.DataFrame({
    "word": word_list_clean
})
print(df_words.head())
print(df_words.shape)
# 统计词频
df_words = (
    df_words
    .groupby("word")["word"]
    .agg(count="size")
    .reset_index()
    .sort_values(by="count", ascending=False)
)
print(df_words.head(10))
# 5. 和单词词典实现merge
df_merge = pd.merge(
    left = df_dict,
    right = df_words,
    left_on = "word",
    right_on = "word"
)
print(df_merge.sample(10))
print(df_merge.shape)
# 6. 存入Excel

# df_merge.to_excel("./39. pdf_chinese_english.xlsx", index=False)
